/**
 * 
 */
package com.gragra.stats;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.objects.Object2DoubleOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
import it.unimi.dsi.fastutil.objects.ObjectSet;

/**
 * @author Christoph Teichmann
 * 16.06.2013
 */
public class WordToTagEntropy
{
	/**
	 * 
	 */
	public static void main(String... args) throws IOException
	{
		Object2DoubleOpenHashMap<String> wordCounts = new Object2DoubleOpenHashMap<String>();
		Object2ObjectOpenHashMap<String, ObjectSet<String>> seenTagsForWords = new Object2ObjectOpenHashMap<String, ObjectSet<String>>();
		BufferedReader in = new BufferedReader(new FileReader(args[0]));
		String line;
		while((line = in.readLine()) != null)
		{add(wordCounts,seenTagsForWords,line.trim().split("\\s+"));}
		in.close();
		Int2DoubleOpenHashMap countCounts = new Int2DoubleOpenHashMap();
		Int2DoubleOpenHashMap countTagCounts = new Int2DoubleOpenHashMap();
		ObjectIterator<String> oit = wordCounts.keySet().iterator();
		while(oit.hasNext())
		{
			String s = oit.next();
			int count = ((int) wordCounts.getDouble(s))/100;
			countCounts.addTo(count, 1.0);
			int amount = seenTagsForWords.get(s).size();
			countTagCounts.addTo(count, amount);
		}
		BufferedWriter bw = new BufferedWriter(new FileWriter(args[1]));
		IntIterator iit = countCounts.keySet().iterator();
		boolean first = true;
		while(iit.hasNext())
		{
			if(first)
			{first = false;}
			else
			{bw.newLine();}
			int count = iit.nextInt();
			bw.write(Integer.toString(count));
			bw.write("\t");
			bw.write(Double.toString(countTagCounts.get(count)/countCounts.get(count)));
			bw.write("\t");
			bw.write(Double.toString(countCounts.get(count)));
			bw.write("\t");
			bw.write(Double.toString(countTagCounts.get(count)));
		}
		bw.close();
	}
	/**
	 * 
	 * @param wordCounts
	 * @param seenTagsForWords
	 * @param split
	 */
	private static void add(
			Object2DoubleOpenHashMap<String> wordCounts,
			Object2ObjectOpenHashMap<String, ObjectSet<String>> seenTagsForWords,
			String[] split)
	{
		for(String s : split)
		{
			String[] pair = s.split("\\|");
			wordCounts.addTo(pair[0], 1.0);
			ObjectSet<String> set = seenTagsForWords.get(pair[0]);
			if(set == null)
			{
				set = new ObjectOpenHashSet<String>();
				seenTagsForWords.put(pair[0], set);
			}
			set.add(pair[1]);
		}
	}
}